# -*- coding: utf-8 -*-

import random
import base64
from settings import USER_AGENTS
from selenium import webdriver
from scrapy.conf import settings
# from scrapy.http.response import Response
from scrapy.http import HtmlResponse
import time
from scrapy import signals
from scrapy.xlib.pydispatch import dispatcher
from telnetlib import DO
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys

chrome_options = Options()
#加上下面两行，解决报错 禁止使用sandbox
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')


class RandomUserAgent(object):

    def process_request(self, request, spider):
        useragent = random.choice(USER_AGENTS)
        request.headers.setdefault("User-Agent", useragent)



# 动态加载浏览器数据
class JavaScriptMiddleware(object):
    def __init__(self):
        self.driver = webdriver.Chrome(executable_path="/usr/local/chromedriver/chromedriver",chrome_options=chrome_options)
        dispatcher.connect(self.spider_closed, signals.spider_closed)

    def process_request(self, request, spider):
        self.driver.get(request.url)
        print("页面渲染中····开始自动下拉页面")
        indexPage = 1000
        while indexPage < self.driver.execute_script("return document.body.offsetHeight"):
            self.driver.execute_script("scroll(0," + str(indexPage) + ")")
            indexPage = indexPage + 1000
            print(indexPage)
            time.sleep(1)

        rendered_body = self.driver.page_source
        # 编码处理
        if r'charset="GBK"' in rendered_body or r'charset=gbk' in rendered_body:
            coding = 'gbk'
        else:
            coding = 'utf-8'
        return HtmlResponse(request.url, body=rendered_body, encoding='utf-8')

    # 关闭浏览器
    def spider_closed(self, spider, reason):
        print ('close driver......')
        self.driver.close()